
# coding: utf-8
import sys, os
from time import sleep

BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASE_PATH)# 项目目录
import requests
from ip_common.headers import get_header
from bs4 import BeautifulSoup
from ip_common import log
from ip_common import proxies
from ip_save import save_local
class xila(object):
    def __init__(self,ip_attr,header,nums,is_proxy=False):
        self.base_url = "http://www.xiladaili.com"
        self.locaton = {
            "gaoni":"/gaoni/"
        }
        self.pages = nums
        self.url = self.base_url+self.locaton[ip_attr]
        self.header = header
        self.is_proxy = is_proxy
        self.data = []
    def get(self):
        for item in self.pages:
            temp_url = self.url + str(item)+"/"
            print(temp_url)
            response = None
            if self.is_proxy:
                proxy = proxies.get_local_proxy_random()
                response = requests.get(url=temp_url, headers = self.header,proxies=proxy)
            else :
                response = requests.get(url=temp_url, headers=self.header)
            content = response.text

            self.data.append( self.parse(content))
            sleep(3)
        return self.data
    def parse(self,text):
        res = {"ip":[]}
        ip_data = {"http":"","ip":"","port":""}
        bf=BeautifulSoup(text,features="lxml")
        page_table = bf.findAll("table")
        if len(page_table) != 1:
            log.error("页面异常，检查链接！")
            return
        page_tr = page_table[0].findAll("tr")[1:]
        if len(page_tr) < 1:
            log.error("表格异常，检查链接！")
            return
        for tr in page_tr:
            # 不是HTTPS
            if tr.findAll("td")[1].string.find("HTTPS")== -1:
                ip_data["http"] = "http"
                temp_str = tr.findAll("td")[0].string
                ip_data["ip"] = temp_str[0:temp_str.find(":")]
                ip_data["port"] = temp_str[temp_str.find(":")+1:]
                res["ip"].append(ip_data.copy())
        return res
    def save(self,loc,model):
        if loc == "local":
            save_local.save_xila(self.data,model)
if __name__=="__main__":

    temp = xila("gaoni",get_header("xila"),[2,3,4,5],False)
    res = temp.get()
    temp.save("local","w")
    print(res)